package hn.cch.flink.demo;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

/**
 * 分词器
 */
public class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {

    @Override
    public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
        // 规范化并分割行
        String[] words = value.toLowerCase().split("\\W+");
        // 为每个单词发出 Tuple2<String, Integer>(word, 1)
        for (String word : words) {
            if (!word.isEmpty()) {
                out.collect(new Tuple2<>(word, 1));
            }
        }
    }
}
